package me.crawler;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.StringTokenizer;
import java.util.regex.Pattern;

public class CrawlerThread extends Thread {
	protected int level;
	protected int id;
	protected URLQueue queue;
	protected CrawlerThreadController tc;
	public void setId(int _id) {
		id = _id;
	}
	public void setLevel(int _level) {
		level = _level;
	}
	public void setQueue(URLQueue _queue) {
		queue = _queue;
	}
	public void setThreadController(CrawlerThreadController _tc) {
		tc = _tc;
	}

	public CrawlerThread() {
	}
	public void run() {
		// pop new urls from the queue until queue is empty
		while (!queue.isEmpty()){
			process(queue.pop(level));

			// If there are less threads running than it could, try
			// starting more threads
			if (tc.getMaxThreads() > tc.getRunningThreads()) {
				try {
					tc.startThreads();
				} catch (Exception e) {
					System.err.println("[" + id + "] " + e.toString());
				}
			}
		}

		tc.finished(id);
	}

	
	
    Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpeg"
            + "|png|tiff?|mid|mp2|mp3|mp4"
            + "|wav|avi|mov|mpeg|ram|m4v|pdf"
            + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

    
    
    
    public boolean validUrl(String url){
		
        String link = url.toLowerCase();
        if (filters.matcher(link).matches()) 
                return false;
        
        if (link.startsWith("wiki"))
            return true;
        
        if(link.length()>3){
        	String dot = link.substring(3, 4);
        if(dot.contains("."))
        	return false;
        }
        
        if (!link.contains("?")) 
            return true;
        
        return false;
    }

	/**
	 * The thread invokes the process method for each object in the queue
	 */
	public void process(Object o){

		URL url = (URL)o;
	
		System.out.println("searching " + url);
    	
		try {
			// try opening the URL
			URLConnection urlConnection = url.openConnection();
	
			//urlConnection.setAllowUserInteraction(false);
	
			InputStream urlStream = url.openStream();
			byte b[] = new byte[1000];
			int numRead = urlStream.read(b);
			String content = new String(b, 0, numRead);
				
			while (numRead != -1) {
	
			    numRead = urlStream.read(b);
			    if (numRead != -1) {
				String newContent = new String(b, 0, numRead);
				content += newContent;
			    }
			}
			urlStream.close();
	
			String lowerCaseContent = content.toLowerCase();
	
			int index = 0;
			while ((index = lowerCaseContent.indexOf("<a", index)) != -1)
			{
			    if ((index = lowerCaseContent.indexOf("href", index)) == -1) 
				break;
			    if ((index = lowerCaseContent.indexOf("=", index)) == -1) 
				break;
			    if ((index = lowerCaseContent.indexOf("/", index)) == -1) 
				break;
			    index++;
			    String remaining = content.substring(index);
			    
			    
	
			    StringTokenizer st  = new StringTokenizer(remaining, "\'\t\n\r\">#");
			    String strLink = st.nextToken();
			    String link = null;
			    
			     
			    try {
			    	
			    	
			    	URL	urlLink = new URL(strLink);	    		
			    	
				    	// link = urlLink.toString();
					//System.out.println(strLink);

	
				    if (validUrl(link))
				    	queue.push(urlLink, level);
				    
				    if (queue.full())
				    	break;
			    
			
			    } catch (MalformedURLException e) {
			    	System.err.println("ERROR: bad URL " + link);
			    	//errorCount++;
			    	//even though we get an error accessing the link we want to continue searching rest of the Queue
			    	continue;
			    }
			}
			}
		    catch (IOException e) {
		    	System.out.println("ERROR: couldn't open URL " + url);
			
		    }
	
		   
		}
	

}
